By: Allyson Lynch

import pandas as pd

Create a file that counts the occurances of each amines

def shorten(stdinchikey):
    inchikey = stdinchikey[:23]
    return inchikey

def create_popularity_count(smiles_file, file_name):
    """takes files of smiles strings by category and returns a file that counts occurances by category of a smiles string"""
    final_df = pd.read_table(smiles_file, index_col=False)
    final_df['inchikey'] = final_df['stdinchikey'].apply(shorten)
    final_df.drop(['[REFCODE]', '[_chemical_name_systematic]', 'Amine', 'stdinchikey'], axis=1, inplace=True) #drop extra initialization columns
    oxides = ["oxides", "metaloxides", "metalborates", "metalformates", "metalhalides", "metaloxalates", "metalperoxides", "metalphosphonates"]
    for category in oxides:
        category_df = pd.read_table("Outputs/01 CCDC Smiles File/Amines_" + category + "_smiles.tsv")
        category_df['inchikey'] = category_df['stdinchikey'].apply(shorten)
        count_df = pd.DataFrame(category_df.groupby('inchikey')['[REFCODE]'].count()) #count occurances
        count_df.rename(columns={'[REFCODE]':"number_" + category}, inplace=True) #name column by category
        count_df['inchikey'] = count_df.index.tolist()
        final_df = pd.merge(final_df, count_df, how='outer', on='inchikey') #merge new dataframe number column onto unique smiles dataframe
    clean_df = final_df.dropna(subset=['smiles']).fillna(0).drop_duplicates().set_index('smiles')
    clean_df.to_csv(file_name, sep='\t') #save as tsv

Assign amine popularity based on a pareto split

def popularity_file(smiles_file, new_file, pareto_split=False):
    """takes a file with smiles strings and returns a new file with smiles and popularity"""
    smiles_df = pd.read_table(smiles_file)
    final_df = pd.DataFrame({'smiles':smiles_df['smiles'].tolist(), 'inchikey':smiles_df['inchikey'].tolist()}, index=smiles_df['smiles'].tolist()) #initiliaze dataframe
    pareto = smiles_df.columns[2:].tolist() #initialize lists
    percent_amines = ["NAN" for mlc in range(len(smiles_df.columns[2:].tolist()))]
    percent_structures = ["NAN" for mlc in range(len(smiles_df.columns[2:].tolist()))]
    gini_values = ["NAN" for mlc in range(len(smiles_df.columns[2:].tolist()))]
    populars = []
    unpopulars = []
    unique = []
    for molecule in smiles_df.columns[2:]: 
        #make a dataframe per molcule (skip smiles)
        molecule_df = smiles_df.loc[:, ['smiles', molecule]] #only include smiles and molecule number
        molecule_df[molecule[7:]+'_probability'] = molecule_df[molecule]/molecule_df[molecule].sum() #add probability column
        probability_df = molecule_df.sort_values(by=[molecule[7:]+'_probability']) #sort by probability in increasing order
        probabilities = probability_df[molecule[7:]+'_probability'].tolist() #list probabilities
        probability_df[molecule[7:]+'_cumulativeprob'] = [(probabilities[i] + sum(probabilities[:i])) for i in range(len(probabilities))]
        probability_list = [x for x in probabilities if x != 0] #remove molecules that are not present
        cumulative_list = [x for x in probability_df[molecule[7:]+'_cumulativeprob'].tolist() if x != 0] #remove molecules that are not present 
        #calculate popularity with pareto method 
        popularity = [] #initialize
        ranks = []
        zeros = []
        rank=0
        for i in range(1, len(probabilities)+1): 
            if probabilities[i-1]==0:
                popularity.append(-1) #absent
                zeros.append(0)
            else:
                rank += 1
                proportion = rank/(len(probability_list)+1) #cacluate rank proportion
                ranks.append(rank)
                probability = probability_list[rank-1] #calculate probabilty
                cumulative = cumulative_list[rank-1] #calculate current cumulative probability
                if probability==probability_list[rank-2]:
                    popularity.append(popularity[-1]) #same popularity for same number
                elif proportion <= (1-cumulative):
                    popularity.append(0) #unpopular
                else:
                    popularity.append(1) #popular
                if pareto_split: #compute without regard to amines with the same count
                    if proportion <= (1-cumulative):
                        i = pareto.index(molecule) #retain only last value
                        percent_amines[i] = proportion
                        percent_structures[i] = cumulative 
                        gini_values[i] = (proportion*2)-1
        #make file
        probability_df[molecule[7:]+'_popular'] = popularity #add popularity column
        count_df = pd.DataFrame(probability_df.groupby(molecule[7:]+'_popular')[molecule[7:]+'_popular'].count())
        count_df['popularity'] = count_df.index.tolist()
        up = count_df.loc[count_df['popularity']==0]
        pp = count_df.loc[count_df['popularity']==1]
        unpopulars.append(up.iloc[0][molecule[7:]+'_popular'])
        populars.append(pp.iloc[0][molecule[7:]+'_popular'])
        unique.append(len(ranks))
        #file per molecule of rank, smiles, number of occurences, fractional number of occurences, popularity
        probability_df['rank'] = zeros + ranks
        probability_df[molecule[7:]+'_proportion'] = probability_df['rank']/(len(probability_list)+1)
        rprobabilities = probability_list[::-1]
        probability_df[molecule[7:]+'_rcumulativeprob'] = zeros + [(rprobabilities[i] + sum(rprobabilities[:i])) for i in range(len(rprobabilities))]
        additional_df = probability_df.loc[probability_df[molecule[7:]+'_popular']>=0] #exclude absent
        additional_df = additional_df.loc[:, ['rank', 'smiles', molecule, molecule[7:]+'_probability', molecule[7:]+'_popular', molecule[7:]+'_rcumulativeprob', molecule[7:]+'_proportion']] #only include molecule
        additional_df.set_index('rank', inplace=True)
        additional_df.to_csv("Outputs/02 Popularity Files/"+molecule[7:]+'_popularity.tsv', sep='\t') #save as tsv
        probability_df.drop(['rank', molecule[7:]+'_proportion', molecule[7:]+'_rcumulativeprob'], axis=1, inplace=True) #drop additional columns
        probability_df.drop([molecule, molecule[7:]+'_probability', molecule[7:]+'_cumulativeprob'], axis=1, inplace=True) #drop number columns
        final_df = pd.merge(probability_df, final_df, left_on='smiles', right_on='smiles', how='outer') #merge new data frame with existing on smiles
    if pareto_split:
        pareto_df = pd.DataFrame({'molecule': smiles_df.columns[2:].tolist(), 'amines_unique':unique, 'percent_amines': percent_amines, 'percent_structures': percent_structures, 'number_popular': populars, 'number_unpopular': unpopulars, 'gini_coefficient':gini_values}).set_index('molecule')
        pareto_df.to_csv("Outputs/02 Popularity Files/pareto_split.tsv", sep='\t') #save as tsv
    final_df = final_df.set_index('smiles') #make smiles index
    final_df.to_csv(new_file, sep='\t') #save as tsv

Combine pareto split information with K-S values

def additional_info(pareto_file, KS_file):
    KS_df = pd.read_table(KS_file)
    KS_df = KS_df.reindex(index=KS_df.index[::-1])
    pareto_df = pd.read_table(pareto_file, index_col='molecule')
    totals = []
    for category in pareto_df.index.tolist():
        category_df = pd.read_table("Outputs/01 CCDC Smiles File/Amines_" + category[7:] + "_smiles.tsv")
        totals.append(len(category_df['smiles'].tolist()))
    if len(pareto_df.columns)>7:
        pareto_df.update({'total_structures':totals})
    else:
        pareto_df.insert(loc=0, column='total_structures', value=totals)
    pareto_df.to_csv("Outputs/05 K-S Test/pareto_CS.tsv", sep='\t') #save as tsv

Calculate the popularity of structures

def structure_popularity():
    """takes a file of outcome success data and popularity and returns a file that calculates a number success per amine, probability, and bootstrap error"""
    structure_df = pd.read_table("Outputs/01 CCDC Smiles File/Amines_oxides_smiles.tsv") #read file
    structure_df['inchikey'] = structure_df['stdinchikey'].apply(shorten)
    popularity_df = pd.read_table("Outputs/02 Popularity Files/amine_popularity.tsv")
    structure_popularity = pd.merge(structure_df, popularity_df, on='inchikey').fillna(-1) #merge popularity with structures
    final_df = pd.DataFrame({'popularity':['Popular', 'NotPopular', 'Unpopular', 'Absent']})
    for oxide in ["metaloxides_popular", "metalborates_popular"]:
        structure_popularity.sort_values(by=[oxide], inplace=True) #sort by popularity (only keep highest amine rank)
        structure_popularity.drop_duplicates(subset='[REFCODE]', keep='first', inplace=True) #deduplicate
        structure_popularity.to_csv("Outputs/02 Popularity Files/structure_popularity_match" + oxide + ".tsv", sep='\t') #save as tsv
        p = structure_popularity.loc[(structure_popularity[oxide]==1)]
        n = structure_popularity.loc[(structure_popularity[oxide]<=0)]
        u = structure_popularity.loc[(structure_popularity[oxide]==0)]
        a = structure_popularity.loc[(structure_popularity[oxide]==-1)] 
        final_df['number_structure_'+oxide[:-9]] = [len(p.index.tolist()), len(n.index.tolist()), len(u.index.tolist()), len(a.index.tolist())] #count
    final_df.to_csv("Outputs/02 Popularity Files/structure_popularity.tsv", sep='\t') #save as tsv
    
structure_popularity()

def popularity_files(pareto_split=False, additional_pareto=False):
    create_popularity_count("Outputs/01 CCDC Smiles File/Amines_oxides_smiles.tsv", "Outputs/02 Popularity Files/smiles_count.tsv")
    popularity_file("Outputs/02 Popularity Files/smiles_count.tsv", "Outputs/02 Popularity Files/amine_popularity.tsv", pareto_split)
    if additional_pareto: #must run K-S first
        additional_info("Outputs/02 Popularity Files/pareto_split.tsv", "Outputs/05 K-S Test/K-S_outcome_category.tsv")
    
popularity_files(pareto_split=True, additional_pareto=True)